require 'rubygems'
require 'mechanize'
require 'json'
require 'xml'

STDOUT.sync=true;

codelist = ['ACCT', 'BIEN','BTEC','CBME','CENG','CHEM','CIEM','CIVL','COMP','CSIT',
  'ECON','EEMT','EESM','ELEC','ENEG','ENGG','ENVR','ENVS','EVNG','EVSM',
  'FINA','FYTG','GBUS','GNED','HART','HLTH','HUMA','IBTM','IELM','ISOM',
  'JEVE','LABU','LAGR','LANG','LIFS','MAFS','MALS','MARK','MATH','MECH',
  'MESF','MGMT','NANO','PDEV','PHYS','RMBI','SBMT','SCED','SCIE','SHSS','SOSC','SSMA','TEMG','UROP']

def TheNode(name, content)
  n = XML::Node.new(name)
  n << content
  return n
end

agent = Mechanize.new
count = 0
codelist.each do |cl|
  
  if(count < 32)
    count = count + 1
    next
  end
  
url = 'https://w5.ab.ust.hk/wcq/cgi-bin/1210/subject/'+cl
page = agent.get(url)

allcourses = page.search('[class="course"]')

doc = XML::Document.new
doc.root = XML::Node.new(cl)

allcourses.each do |c|
  courseTag = c.search('h2')[0].text
  courseCode = ((/\w+\s+\w+\s+-/.match(courseTag))[0])[0..-2].strip
  puts courseCode
  courseName = ((/-.+\(/.match(courseTag))[0])[1..-2].strip
  courseUnit = /\(\d/.match(courseTag)[0][1..-1]
  
  course = XML::Node.new('course')
  course['id'] = courseCode
  cName = TheNode('courseName',courseName)
  cUnit = TheNode('courseUnit',courseUnit)
  course<<cName
  course<<cUnit
  
  courseContent = c.search('[class="newsect secteven"]') + c.search('[class="newsect sectodd"]')
  
  courseContent.each do |cc|
    cc_td = cc.search('td')   
    sect = ((/\w+\s+\(/.match(cc_td[0].text))[0])[0..-2].strip
    
    datetime = /[Mo|Tu|We|Th|Fr]+\s\d\d:\d\d\w\w\s-\s\d\d:\d\d\w\w/.match(cc_td[1].text)
    if(datetime==nil)
      datetime = 'TBA'
    else
      datetime = datetime[0]
    end
    
    cSect = XML::Node.new('section')
    cSect['id'] = sect
    cSect << TheNode('dateTime',datetime)
    cSect << TheNode('place',cc_td[2].text)
    cProfs = XML::Node.new('profs')
    if(cc_td[3].search('a').length==0)
      cProfs << TheNode('prof',cc_td[3].text)
    else
      cc_td[3].search('a').each do |p|
        cProfs << TheNode('prof',p.text)
      end
    end
    cSect << cProfs
    course << cSect
  end
  doc.root << course
end
  puts cl+' '+count.to_s
  doc.save(cl+'.xml', :indent=>true, :encoding => XML::Encoding::UTF_8)
  count = count + 1
end